
<!doctype html>
<html lang="en" class="no-js">
  <head>
      <!-- Google tag (gtag.js) -->
      <script async src="https://www.googletagmanager.com/gtag/js?id=G-7QRVV2J34B"></script>
      <script>
        window.dataLayer = window.dataLayer || [];
        function gtag(){dataLayer.push(arguments);}
        gtag('js', new Date());

        gtag('config', 'G-7QRVV2J34B');
      </script>
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">
      
      
      
      
        <link rel="prev" href="../operations/">
      
      
        <link rel="next" href="../tutorials/">
      
      
      <link rel="icon" href="../icon.png">
      <meta name="generator" content="mkdocs-1.6.0, mkdocs-material-9.5.29">
    
    
      
        <title>Layers - JS-PyTorch</title>
      
    
    
      <link rel="stylesheet" href="../assets/stylesheets/main.76a95c52.min.css">
      
        
        <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
      
      


    
    
      
    
    
      
        
        
        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
      
    
    
      <link rel="stylesheet" href="../style.css">
    
    <script>__md_scope=new URL("..",location),__md_hash=e=>[...e].reduce((e,_)=>(e<<5)-e+_.charCodeAt(0),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
    
      

    
    
    
  </head>
  
  
    
    
      
    
    
    
    
    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="deep-orange" data-md-color-accent="deep-orange">
  
    
    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" for="__drawer"></label>
    <div data-md-component="skip">
      
        
        <a href="#layers" class="md-skip">
          Skip to content
        </a>
      
    </div>
    <div data-md-component="announce">
      
    </div>
    
    
      

  

<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
  <nav class="md-header__inner md-grid" aria-label="Header">
    <a href="https://github.com/eduardoleao052/js-pytorch" title="JS-PyTorch" class="md-header__button md-logo" aria-label="JS-PyTorch" data-md-component="logo">
      
  <img src="../white_icon.png" alt="logo">

    </a>
    <label class="md-header__button md-icon" for="__drawer">
      
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z"/></svg>
    </label>
    <div class="md-header__title" data-md-component="header-title">
      <div class="md-header__ellipsis">
        <div class="md-header__topic">
          <span class="md-ellipsis">
          </span>
        </div>
        <div class="md-header__topic" data-md-component="header-topic">
          <span class="md-ellipsis">
            
              Layers
            
          </span>
        </div>
      </div>
    </div>
    
      
<form class="md-header__option" data-md-component="palette">
  
    
    
    
    <input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="deep-orange" data-md-color-accent="deep-orange"  aria-label="Switch to dark mode"  type="radio" name="__palette" id="__palette_0">
    
      <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_1" hidden>
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h10a5 5 0 0 0 5-5 5 5 0 0 0-5-5m0 8a3 3 0 0 1-3-3 3 3 0 0 1 3-3 3 3 0 0 1 3 3 3 3 0 0 1-3 3Z"/></svg>
      </label>
    
  
    
    
    
    <input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="deep-orange" data-md-color-accent="deep-orange"  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_1">
    
      <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_0" hidden>
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 6H7c-3.31 0-6 2.69-6 6s2.69 6 6 6h10c3.31 0 6-2.69 6-6s-2.69-6-6-6zm0 10H7c-2.21 0-4-1.79-4-4s1.79-4 4-4h10c2.21 0 4 1.79 4 4s-1.79 4-4 4zM7 9c-1.66 0-3 1.34-3 3s1.34 3 3 3 3-1.34 3-3-1.34-3-3-3z"/></svg>
      </label>
    
  
</form>
      
    
    
      <script>var media,input,key,value,palette=__md_get("__palette");if(palette&&palette.color){"(prefers-color-scheme)"===palette.color.media&&(media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']"),palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent"));for([key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
    
    
    
      <label class="md-header__button md-icon" for="__search">
        
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
      </label>
      <div class="md-search" data-md-component="search" role="dialog">
  <label class="md-search__overlay" for="__search"></label>
  <div class="md-search__inner" role="search">
    <form class="md-search__form" name="search">
      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
      <label class="md-search__icon md-icon" for="__search">
        
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z"/></svg>
        
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
      </label>
      <nav class="md-search__options" aria-label="Search">
        
        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
          
          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z"/></svg>
        </button>
      </nav>
      
    </form>
    <div class="md-search__output">
      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
        <div class="md-search-result" data-md-component="search-result">
          <div class="md-search-result__meta">
            Initializing search
          </div>
          <ol class="md-search-result__list" role="presentation"></ol>
        </div>
      </div>
    </div>
  </div>
</div>
    
    
      <div class="md-header__source">
        <a href="https://github.com/eduardoleao052/js-pytorch" title="Go to repository" class="md-source" data-md-component="source">
  <div class="md-source__icon md-icon">
    
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.5.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
  </div>
  <div class="md-source__repository">
    JS-PyTorch
  </div>
</a>
      </div>
    
  </nav>
  
    
      
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
  <div class="md-grid">
    <ul class="md-tabs__list">
      
        
  
  
  
    <li class="md-tabs__item">
      <a href=".." class="md-tabs__link">
        
  
    
  
  Home

      </a>
    </li>
  

      
        
  
  
  
    <li class="md-tabs__item">
      <a href="../tensor/" class="md-tabs__link">
        
  
    
  
  Tensor

      </a>
    </li>
  

      
        
  
  
  
    <li class="md-tabs__item">
      <a href="../operations/" class="md-tabs__link">
        
  
    
  
  Operations

      </a>
    </li>
  

      
        
  
  
    
  
  
    <li class="md-tabs__item md-tabs__item--active">
      <a href="./" class="md-tabs__link">
        
  
    
  
  Layers

      </a>
    </li>
  

      
        
  
  
  
    <li class="md-tabs__item">
      <a href="../tutorials/" class="md-tabs__link">
        
  
    
  
  Tutorials

      </a>
    </li>
  
    <li class="md-tabs__item">
      <a href="../demo/" class="md-tabs__link">
        
  
    
  
  Demo

      </a>
    </li>
      
    </ul>
  </div>
</nav>
    
  
</header>
    
    <div class="md-container" data-md-component="container">
      
      
        
      
      <main class="md-main" data-md-component="main">
        <div class="md-main__inner md-grid">
          
            
              
              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    


  


  

<nav class="md-nav md-nav--primary md-nav--lifted md-nav--integrated" aria-label="Navigation" data-md-level="0">
  <label class="md-nav__title" for="__drawer">
    <a href="https://github.com/eduardoleao052/js-pytorch" title="JS-PyTorch" class="md-nav__button md-logo" aria-label="JS-PyTorch" data-md-component="logo">
      
  <img src="../white_icon.png" alt="logo">

    </a>
    JS-PyTorch
  </label>
  
    <div class="md-nav__source">
      <a href="https://github.com/eduardoleao052/js-pytorch" title="Go to repository" class="md-source" data-md-component="source">
  <div class="md-source__icon md-icon">
    
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.5.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
  </div>
  <div class="md-source__repository">
    JS-PyTorch
  </div>
</a>
    </div>
  
  <ul class="md-nav__list" data-md-scrollfix>
    
      
      
  
  
  
  
    <li class="md-nav__item">
      <a href=".." class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Home
  </span>
  

      </a>
    </li>
  

    
      
      
  
  
  
  
    <li class="md-nav__item">
      <a href="../tensor/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Tensor
  </span>
  

      </a>
    </li>
  

    
      
      
  
  
  
  
    <li class="md-nav__item">
      <a href="../operations/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Operations
  </span>
  

      </a>
    </li>
  

    
      
      
  
  
    
  
  
  
    <li class="md-nav__item md-nav__item--active">
      
      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
      
      
        
      
      
        <label class="md-nav__link md-nav__link--active" for="__toc">
          
  
  <span class="md-ellipsis">
    Layers
  </span>
  

          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <a href="./" class="md-nav__link md-nav__link--active">
        
  
  <span class="md-ellipsis">
    Layers
  </span>
  

      </a>
      
        

<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
  
  
  
    
  
  
    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      Table of contents
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#nnlinear" class="md-nav__link">
    <span class="md-ellipsis">
      nn.Linear
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#nnmultiheadselfattention" class="md-nav__link">
    <span class="md-ellipsis">
      nn.MultiHeadSelfAttention
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#nnfullyconnected" class="md-nav__link">
    <span class="md-ellipsis">
      nn.FullyConnected
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#nnblock" class="md-nav__link">
    <span class="md-ellipsis">
      nn.Block
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#nnembedding" class="md-nav__link">
    <span class="md-ellipsis">
      nn.Embedding
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#nnpositionalembedding" class="md-nav__link">
    <span class="md-ellipsis">
      nn.PositionalEmbedding
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#nnrelu" class="md-nav__link">
    <span class="md-ellipsis">
      nn.ReLU
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#nnsoftmax" class="md-nav__link">
    <span class="md-ellipsis">
      nn.Softmax
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#nndropout" class="md-nav__link">
    <span class="md-ellipsis">
      nn.Dropout
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#nnlayernorm" class="md-nav__link">
    <span class="md-ellipsis">
      nn.LayerNorm
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#nncrossentropyloss" class="md-nav__link">
    <span class="md-ellipsis">
      nn.CrossEntropyLoss
    </span>
  </a>
  
</li>
      
    </ul>
  
</nav>
      
    </li>
  

    
      
      
  
  
  
  
    <li class="md-nav__item">
      <a href="../tutorials/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Tutorials
  </span>
  

      </a>
    </li>
  
    <li class="md-nav__item">
      <a href="../demo/" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Demo
  </span>
  

      </a>
    </li>
    
  </ul>
</nav>
                  </div>
                </div>
              </div>
            
            
          
          
            <div class="md-content" data-md-component="content">
              <article class="md-content__inner md-typeset">
                
                  

  
  


<h1 id="layers">Layers</h1>
<p>In this section are listed all of the <strong>Layers</strong> and <strong>Modules</strong>.</p>
<h2 id="nnlinear">nn.Linear</h2>
<pre><code>new nn.Linear(in_size,
              out_size,
              device, 
              bias, 
              xavier)
</code></pre>
<p>Applies a linear transformation to the input tensor.
Input is matrix-multiplied by a <code>w</code> tensor and added to a <code>b</code> tensor.</p>
<p>Parameters</p>
<ul>
<li><strong>in_size (number)</strong> - Size of the last dimension of the input data.</li>
<li><strong>out_size (number)</strong> - Size of the last dimension of the output data.</li>
<li><strong>device (string)</strong> - Device on which the model's calculations will run. Either <code>'cpu'</code> or <code>'gpu'</code>.</li>
<li><strong>bias (boolean)</strong> - Whether to use a bias term <code>b</code>.</li>
<li><strong>xavier (boolean)</strong> - Whether to use Xavier initialization on the weights.</li>
</ul>
<p>Learnable Variables</p>
<ul>
<li><strong>w</strong> - <em>[input_size, output_size]</em> Tensor.</li>
<li><strong>b</strong> - <em>[output_size]</em> Tensor.</li>
</ul>
<p>Example</p>
<pre><code class="language-javascript">&gt;&gt;&gt; const linear = new nn.Linear(10,15,'gpu');
&gt;&gt;&gt; let x = torch.randn([100,50,10], true, 'gpu');
&gt;&gt;&gt; let y = linear.forward(x);
&gt;&gt;&gt; y.shape
// [100, 50, 15]
</code></pre>
<p></br></p>
<h2 id="nnmultiheadselfattention">nn.MultiHeadSelfAttention</h2>
<pre><code>new nn.MultiHeadSelfAttention(in_size,
                              out_size,
                              n_heads,
                              n_timesteps,
                              dropout_prob,
                              device)
</code></pre>
<p>Applies a self-attention layer on the input tensor.</p>
<ul>
<li>Matrix-multiplies input by <code>Wk</code>, <code>Wq</code>, <code>Wv</code>, resulting in Key, Query and Value tensors.</li>
<li>Computes attention multiplying Query and transpose Key.</li>
<li>Applies Mask, Dropout and Softmax to attention activations.</li>
<li>Multiplies result by Values.</li>
<li>Multiplies result by <code>residual_proj</code>.</li>
<li>Applies final Dropout.</li>
</ul>
<p>Parameters</p>
<ul>
<li><strong>in_size (number)</strong> - Size of the last dimension of the input data.</li>
<li><strong>out_size (number)</strong> - Size of the last dimension of the output data.</li>
<li><strong>n_heads (boolean)</strong> - Number of parallel attention heads the data is divided into. In_size must be divided evenly by n_heads.</li>
<li><strong>n_timesteps (boolean)</strong> - Number of timesteps computed in parallel by the transformer.</li>
<li><strong>dropout_prob (boolean)</strong> - probability of randomly dropping an activation during training (to improve regularization).</li>
<li><strong>device (string)</strong> - Device on which the model's calculations will run. Either <code>'cpu'</code> or <code>'gpu'</code>.</li>
</ul>
<p>Learnable Variables</p>
<ul>
<li><strong>Wk</strong> - <em>[input_size, input_size]</em> Tensor.</li>
<li><strong>Wq</strong> - <em>[input_size, input_size]</em> Tensor.</li>
<li><strong>Wv</strong> - <em>[input_size, input_size]</em> Tensor.</li>
<li><strong>residual_proj</strong> - <em>[input_size, output_size]</em> Tensor.</li>
</ul>
<p>Example</p>
<pre><code class="language-javascript">&gt;&gt;&gt; const att = new nn.MultiHeadSelfAttention(10, 15, 2, 32, 0.2, 'gpu');
&gt;&gt;&gt; let x = torch.randn([100,50,10], true, 'gpu');
&gt;&gt;&gt; let y = att.forward(x);
&gt;&gt;&gt; y.shape
// [100, 50, 15]
</code></pre>
<p></br></p>
<h2 id="nnfullyconnected">nn.FullyConnected</h2>
<pre><code>new nn.FullyConnected(in_size,
                      out_size,
                      dropout_prob,
                      device,
                      bias)
</code></pre>
<p>Applies a fully-connected layer on the input tensor.</p>
<ul>
<li>Matrix-multiplies input by Linear layer <code>l1</code>, upscaling the input.</li>
<li>Passes tensor through ReLU.</li>
<li>Matrix-multiplies tensor by Linear layer <code>l2</code>, downscaling the input.</li>
<li>Passes tensor through Dropout.</li>
</ul>
<pre><code class="language-javascript">forward(x: Tensor): Tensor {
    let z = this.l1.forward(x);
    z = this.relu.forward(z);
    z = this.l2.forward(z);
    z = this.dropout.forward(z);
    return z;
}
</code></pre>
<p>Parameters</p>
<ul>
<li><strong>in_size (number)</strong> - Size of the last dimension of the input data.</li>
<li><strong>out_size (number)</strong> - Size of the last dimension of the output data.</li>
<li><strong>dropout_prob (boolean)</strong> - probability of randomly dropping an activation during training (to improve regularization).</li>
<li><strong>device (string)</strong> - Device on which the model's calculations will run. Either <code>'cpu'</code> or <code>'gpu'</code>.</li>
<li><strong>bias (boolean)</strong> - Whether to use a bias term <code>b</code>.</li>
</ul>
<p>Learnable Variables</p>
<ul>
<li><strong>l1</strong> - <em>[input_size, 4input_size]</em> Tensor.</li>
<li><strong>l2</strong> - <em>[4input_size, input_size]</em> Tensor.</li>
</ul>
<p>Example</p>
<pre><code class="language-javascript">&gt;&gt;&gt; const fc = new nn.FullyConnected(10, 15, 0.2, 'gpu');
&gt;&gt;&gt; let x = torch.randn([100,50,10], true, 'gpu');
&gt;&gt;&gt; let y = fc.forward(x);
&gt;&gt;&gt; y.shape
// [100, 50, 15]
</code></pre>
<p></br></p>
<h2 id="nnblock">nn.Block</h2>
<pre><code>new nn.Block(in_size,
             out_size,
             n_heads,
             n_timesteps,
             dropout_prob,
             device)
</code></pre>
<p>Applies a transformer Block layer on the input tensor.</p>
<pre><code class="language-javascript">forward(x: Tensor): Tensor {
    // Pass through Layer Norm and Self Attention:
    let z = x.add(this.att.forward(this.ln1.forward(x)));
    // Pass through Layer Norm and Fully Connected:
    z = z.add(this.fcc.forward(this.ln2.forward(z)));
    return z;
}
</code></pre>
<p>Parameters</p>
<ul>
<li><strong>in_size (number)</strong> - Size of the last dimension of the input data.</li>
<li><strong>out_size (number)</strong> - Size of the last dimension of the output data.</li>
<li><strong>n_heads (boolean)</strong> - Number of parallel attention heads the data is divided into. In_size must be divided evenly by n_heads.</li>
<li><strong>n_timesteps (boolean)</strong> - Number of timesteps computed in parallel by the transformer.</li>
<li><strong>dropout_prob (boolean)</strong> - probability of randomly dropping an activation during training (to improve regularization).</li>
<li><strong>device (string)</strong> - Device on which the model's calculations will run. Either <code>'cpu'</code> or <code>'gpu'</code>.</li>
</ul>
<p>Learnable Modules</p>
<ul>
<li><strong>nn.MultiHeadSelfAttention</strong> - <code>Wk</code>, <code>Wq</code>, <code>Wv</code>, <code>residual_proj</code>.</li>
<li><strong>nn.LayerNorm</strong> - <code>gamma</code>, <code>beta</code>.</li>
<li><strong>nn.FullyConnecyed</strong> - <code>l1</code>, <code>l2</code>.</li>
<li><strong>nn.LayerNorm</strong> - <code>gamma</code>, <code>beta</code>.</li>
</ul>
<p><br></p>
<h2 id="nnembedding">nn.Embedding</h2>
<pre><code>new nn.Embedding(in_size,
                 embed_size)
</code></pre>
<p>Embedding table, with a number of embeddings equal to the vocabulary size of the model <code>in_size</code>, and size of each embedding equal to <code>embed_size</code>. For each element in the input tensor (integer), returns the embedding indexed by the integer.</p>
<pre><code class="language-javascript">forward(idx: Tensor): Tensor {
   // Get embeddings indexed by input (idx):
   let x = this.E.at(idx);
   return x;
}
</code></pre>
<p>Parameters</p>
<ul>
<li><strong>in_size (number)</strong> - Number of different classes the model can predict (vocabulary size).</li>
<li><strong>embed_size (number)</strong> - Dimension of each embedding generated.</li>
</ul>
<p>Learnable Parameters</p>
<ul>
<li><strong>E</strong> - <em>[vocab_size, embed_size]</em> Tensor.</li>
</ul>
<p>Example</p>
<pre><code class="language-javascript">&gt;&gt;&gt; const batch_size = 32;
&gt;&gt;&gt; const number_of_timesteps = 256;
&gt;&gt;&gt; const embed = new nn.Embedding(10, 64);
&gt;&gt;&gt; let x = torch.randint(0, 10, [batch_size, number_of_timesteps]);
&gt;&gt;&gt; let y = embed.forward(x);
&gt;&gt;&gt; y.shape
// [32, 256, 64]
</code></pre>
<p><br></p>
<h2 id="nnpositionalembedding">nn.PositionalEmbedding</h2>
<pre><code>new nn.PositionalEmbedding(input_size,
                           embed_size)
</code></pre>
<p>Embedding table, with a number of embeddings equal to the input size of the model <code>input_size</code>, and size of each embedding equal to <code>embed_size</code>. For each element in the input tensor, returns the embedding indexed by it's position.</p>
<pre><code class="language-javascript">forward(idx: Tensor): Tensor {
   // Get dimension of the input:
   const [B, T] = idx.shape;
   // Gets positional embeddings for each element along &quot;T&quot;: (Batch, Timesteps) =&gt; (Batch, Timesteps, Embed)
   const x = this.E.at([...Array(T).keys()]);
   return x
}
</code></pre>
<p>Parameters</p>
<ul>
<li><strong>input_size (number)</strong> - Number of different embeddings in the lookup table (size of the input).</li>
<li><strong>embed_size (number)</strong> - Dimension of each embedding generated.</li>
</ul>
<p>Learnable Parameters</p>
<ul>
<li><strong>E</strong> - <em>[input_size, embed_size]</em> Tensor.</li>
</ul>
<p>Example</p>
<pre><code class="language-javascript">&gt;&gt;&gt; const batch_size = 32;
&gt;&gt;&gt; const number_of_timesteps = 256;
&gt;&gt;&gt; const embed = new nn.PositionalEmbedding(number_of_timesteps, 64);
&gt;&gt;&gt; let x = torch.randint(0, 10, [batch_size, number_of_timesteps]);
&gt;&gt;&gt; let y = embed.forward(x);
&gt;&gt;&gt; y.shape
// [32, 256, 64]
</code></pre>
<p><br></p>
<h2 id="nnrelu">nn.ReLU</h2>
<pre><code>new nn.ReLU()
</code></pre>
<p>Rectified Linear Unit activation function. This implementation is <strong>leaky</strong> for stability.
For each element in the incoming tensor:</p>
<ul>
<li>If element is positive, no change.</li>
<li>If element is negative, multiply by 0.001.</li>
</ul>
<p>Parameters</p>
<ul>
<li><strong>None</strong></li>
</ul>
<p>Learnable Parameters</p>
<ul>
<li><strong>None</strong></li>
</ul>
<p><br></p>
<h2 id="nnsoftmax">nn.Softmax</h2>
<pre><code>new nn.Softmax()
</code></pre>
<p>Softmax activation function. Rescales the data in the input tensor, along the <code>dim</code> dimension. The sum of every element along this dimension is one, and every element is between zero and one.</p>
<pre><code class="language-javascript">forward(x: Tensor, dim = -1): Tensor {
   z = exp(z);
   const out = z.div(z.sum(dim, true));
   return out;
   return x
}
</code></pre>
<p>Parameters</p>
<ul>
<li><strong>None</strong></li>
</ul>
<p>Learnable Parameters</p>
<ul>
<li><strong>None</strong></li>
</ul>
<p>Example</p>
<pre><code class="language-javascript">&gt;&gt;&gt; const softmax = new nn.Softmax();
&gt;&gt;&gt; let x = torch.randn([2,4]);
&gt;&gt;&gt; let y = softmax.forward(x, -1);
&gt;&gt;&gt; y.data
// [[0.1, 0.2, 0.8, 0.0],
//  [0.6, 0.1, 0.2, 0.1]]
</code></pre>
<p><br></p>
<h2 id="nndropout">nn.Dropout</h2>
<pre><code>new nn.Dropout(drop_prob: number)
</code></pre>
<p>Dropout class. For each element in input tensor, has a <code>drop_prob</code> chance of setting it to zero.</p>
<p>Parameters</p>
<ul>
<li><strong>drop_prob (number)</strong> - Probability to drop each value in input, from 0 to 1.</li>
</ul>
<p>Learnable Parameters</p>
<ul>
<li><strong>None</strong></li>
</ul>
<p>Example</p>
<pre><code class="language-javascript">&gt;&gt;&gt; const dropout = new nn.Dropout(0.5);
&gt;&gt;&gt; let x = torch.ones([2,4]);
&gt;&gt;&gt; let y = dropout.forward(x);
&gt;&gt;&gt; y.data
// [[1, 0, 0, 1],
//  [0, 1, 0, 1]]
</code></pre>
<p><br></p>
<h2 id="nnlayernorm">nn.LayerNorm</h2>
<pre><code>new nn.LayerNorm(n_embed: number)
</code></pre>
<p>LayerNorm class. Normalizes the data, with a <strong>mean of 0</strong> and <strong>standard deviation of 1</strong>, across the last dimension. This is done as described in the <a target="_blank" href="https://arxiv.org/abs/1607.06450">LayerNorm paper</a>.</p>
<p>Parameters</p>
<ul>
<li><strong>n_embed (number)</strong> - Size of the last dimension of the input.</li>
</ul>
<p>Learnable Parameters</p>
<ul>
<li><strong>gamma (number)</strong> - Constant to multiply output by (initialized as 1).</li>
<li><strong>beta (number)</strong> - Constant to add to output (initialized as 0).</li>
</ul>
<p><br></p>
<h2 id="nncrossentropyloss">nn.CrossEntropyLoss</h2>
<pre><code>new nn.CrossEntropyLoss()
</code></pre>
<p>Cross Entropy Loss function. Computes the cross entropy loss between the target and the input tensor.</p>
<ul>
<li>First, calculates softmax of input tensor.</li>
<li>Then, selects the elements of the input corresponding to the correct class in the target.</li>
<li>Gets the negative log of these elements.</li>
<li>Adds all of them, and divides by the number of elemets.</li>
</ul>
<p>Parameters</p>
<ul>
<li><strong>None</strong></li>
</ul>
<p>Learnable Parameters</p>
<ul>
<li><strong>None</strong></li>
</ul>
<p>Example</p>
<pre><code class="language-javascript">&gt;&gt;&gt; const number_of_classes = 10;
&gt;&gt;&gt; const input_size = 64;
&gt;&gt;&gt; const loss_func = new nn.CrossEntropyLoss();
&gt;&gt;&gt; let x = torch.randn([input_size, number_of_classes]);
&gt;&gt;&gt; let y = torch.randint(0, number_of_classes, [input_size]);
&gt;&gt;&gt; let loss = loss_func.forward(x, y);
&gt;&gt;&gt; loss.data
// 2.3091357
</code></pre>












                
              </article>
            </div>
          
          
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
        </div>
        
          <button type="button" class="md-top md-icon" data-md-component="top" hidden>
  
  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12Z"/></svg>
  Back to top
</button>
        
      </main>
      
        <footer class="md-footer">
  
  <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
      <div class="md-copyright">
  
    <div class="md-copyright__highlight">
      Copyright &copy; 2024 Eduardo Leitão da Cunha Opice Leão
    </div>
  
  
    Made with
    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
      Material for MkDocs
    </a>
  
</div>
      
        <div class="md-social">
  
    
    
    
    
      
      
    
    <a href="https://github.com/eduardoleao052/js-pytorch" target="_blank" rel="noopener" title="github.com" class="md-social__link">
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><!--! Font Awesome Free 6.5.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg>
    </a>
  
    
    
    
    
      
      
    
    <a href="https://www.linkedin.com/in/eduardoleao052/" target="_blank" rel="noopener" title="www.linkedin.com" class="md-social__link">
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.5.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M416 32H31.9C14.3 32 0 46.5 0 64.3v383.4C0 465.5 14.3 480 31.9 480H416c17.6 0 32-14.5 32-32.3V64.3c0-17.8-14.4-32.3-32-32.3zM135.4 416H69V202.2h66.5V416zm-33.2-243c-21.3 0-38.5-17.3-38.5-38.5S80.9 96 102.2 96c21.2 0 38.5 17.3 38.5 38.5 0 21.3-17.2 38.5-38.5 38.5zm282.1 243h-66.4V312c0-24.8-.5-56.7-34.5-56.7-34.6 0-39.9 27-39.9 54.9V416h-66.4V202.2h63.7v29.2h.9c8.9-16.8 30.6-34.5 62.9-34.5 67.2 0 79.7 44.3 79.7 101.9V416z"/></svg>
    </a>
  
</div>
      
    </div>
  </div>
</footer>
      
    </div>
    <div class="md-dialog" data-md-component="dialog">
      <div class="md-dialog__inner md-typeset"></div>
    </div>
    
    
    <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.path", "navigation.indexes", "toc.integrate", "toc.follow", "navigation.top"], "search": "../assets/javascripts/workers/search.b8dbb3d2.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
    
    
      <script src="../assets/javascripts/bundle.fe8b6f2b.min.js"></script>
      
    
  </body>
</html>